package org.heydr.common.http;

import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.List;

import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.NameValuePair;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.conn.params.ConnRoutePNames;
import org.apache.http.impl.client.AbstractHttpClient;
import org.apache.http.util.ByteArrayBuffer;
import org.heydr.entity.HttpSnatchParam;
import org.heydr.entity.Page;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * 
 * 
 */
public final class HttpSnatcher {
	private static final Logger log = LoggerFactory.getLogger(HttpSnatcher.class);
	private static AbstractHttpClient httpClient;
	private static EncodingDetector encodingDetector;

	
	static {
		// 初始化成员变量
		httpClient = HttpClientFactory.getHttpClient();
		encodingDetector = new EncodingDetector();
	}
	
	/**
	 * 下载url,得到Page
	 * 
	 * @param httpSnatchParam
	 * @return 失败:null
	 */
	public static Page getPage(HttpSnatchParam httpSnatchParam) {
		// 执行http请求
		HttpResult httpResult = execute(httpSnatchParam);
		if (httpResult == null) {
			return null;
		}
		String finalUrl = httpResult.finalUrl; // 最终的url
		HttpUriRequest httpRequest = httpResult.httpRequest;
		HttpResponse response = httpResult.httpResponse;

		// If the response does not enclose an entity, there is no need
		// to bother about connection release
		HttpEntity responseEntity = response.getEntity(); // 得到http内容实体
		if (null == responseEntity) {
			return null;
		}

		// 得到http响应状态码
		int httpStatus = response.getStatusLine().getStatusCode();

		// http 响应码不为200，返回.
		if (httpStatus != HttpStatus.SC_OK) {
			closeResponse(response, finalUrl); // 关闭响应流,释放连接
			Page page = new Page();
			page.setUrl(httpSnatchParam.getUrl());
			page.setFinalUrl(finalUrl);
			page.setHttpStatus(httpStatus);
			return page;
		}

		String contentType = getContentType(response);
		log.debug("contentType:" + contentType + ",for url:" + finalUrl);
		if (null == contentType || "".equals(contentType)) {
			log.error("no response header: content-type" + ",for url:" + finalUrl);
			httpRequest.abort();// 终止流,不再接收数据，关闭连接
			return null;
		}

		// 得到响应内容流
		InputStream contentStream = null;
		try {
			contentStream = responseEntity.getContent();
			if (contentStream != null) {
				contentStream = new BufferedInputStream(contentStream);
			}
		} catch (Exception e) {
			return null;
		}

		try {
			// 获取content-length
			long contentLength = responseEntity.getContentLength();
			log.debug("contentLength : " + contentLength);

			// 读取网页内容到byte数组
			byte[] content = readToByteArray(contentStream, contentLength);
			contentStream.close(); // 关闭流，释放连接
			if (null == content || 0 == content.length) { // 没有内容
				log.error("not get cotent for url:" + finalUrl);
				return null;
			}

			// 得到字符集
			String charset = encodingDetector.getEncoding(contentType, content);
			log.debug("charset: " + charset + ",for url: " + finalUrl);
			if (null == charset || "".equals(charset)) {
				log.error("not get content charset for url:" + finalUrl);
				return null;
			}

			// 编码内容
			String contentStr = new String(content, charset);

			Page page = new Page();
			page.setUrl(httpSnatchParam.getUrl());
			page.setFinalUrl(finalUrl);
			page.setContent(contentStr);
			page.setHttpStatus(httpStatus);

			return page;
		} catch (Exception e) {
			log.error("url:" + finalUrl);
			log.error(e.getMessage(), e);
		}

		return null;
	}
	
	/**
	 * 执行http请求
	 * 
	 * @param httpSnatchParam
	 *            失败:null
	 */
	private static HttpResult execute(HttpSnatchParam httpSnatchParam) {
		// http参数
		String url = httpSnatchParam.getUrl();
		List<Header> addRequestHeaders = httpSnatchParam.getRequestHeaders(); // 额外请求头
		Proxy proxy = httpSnatchParam.getProxy(); // http代理

		// 构造http请求
		HttpUriRequest httpRequest = constructHttpRequest(httpSnatchParam);
		if (httpRequest == null) {
			return null;
		}

		// 执行http请求
		HttpResponse response = null;
		try {
			response = executeRequest(httpRequest, proxy);
		} catch (Exception e) {
			log.error("url:" + url);
			log.error(e.getMessage(), e);
			return null;
		}

		// 处理重定向，最多追踪n层重定向
		int httpStatus = response.getStatusLine().getStatusCode(); // 得到http响应状态码
		if (httpSnatchParam.isRedirecting()) { // 下载资源url不需要重定向
			for (int i = 1; i <= 10 && ((httpStatus == HttpStatus.SC_MOVED_TEMPORARILY) || (httpStatus == HttpStatus.SC_MOVED_PERMANENTLY) || (httpStatus == HttpStatus.SC_SEE_OTHER) || (httpStatus == HttpStatus.SC_TEMPORARY_REDIRECT)); i++) {
				// 读取新的URL地址
				Header locationHeader = response.getFirstHeader("location");
				if (locationHeader != null) {
					String locationUrl = locationHeader.getValue();
					log.debug("locationUrl:" + locationUrl);
					if (null != locationUrl && !"".equals(locationUrl)) {
						locationUrl = URLUtil.completeUrl(locationUrl, url);
						log.debug("url : " + url + " redirect to ----->url : " + locationUrl);
						url = locationUrl; // 新的url地址
					}
				} else {
					log.error("Invalid redirect for url:" + url);
					closeResponse(response, url); // 关闭响应流,释放连接
					return null;
				}

				// 爬取新的url
				try {
					closeResponse(response, url); // 关闭响应流,释放连接

					// 预处理url
					String changedUrl = StringUtils.hexToStr(url);
					if (httpSnatchParam.isNeedEncodeUrl()) {
						String urlEncoding = httpSnatchParam.getUrlEncoding();
						if (urlEncoding == null || urlEncoding.length() == 0) {
							urlEncoding = "utf-8"; // 默认为"utf-8"
						}
						log.debug("url=" + changedUrl);
						changedUrl = URLUtil.encode(changedUrl, urlEncoding);
						log.debug("encoded url=" + changedUrl);
					}

					// 构造http请求
					httpRequest = new HttpGet(changedUrl);
					// 添加额外请求头
					if (null != addRequestHeaders && addRequestHeaders.size() > 0) {
						for (Header header : addRequestHeaders) {
							httpRequest.addHeader(header);
						}
					}

					// 执行请求，得到响应
					response = executeRequest(httpRequest, proxy);
					httpStatus = response.getStatusLine().getStatusCode();
				} catch (Exception e) {
					log.error("url:" + url);
					log.error(e.getMessage(), e);
					return null;
				}
			}
		}

		// 返回执行结果
		HttpResult httpResult = new HttpResult();
		httpResult.finalUrl = url; // 最终url
		httpResult.httpRequest = httpRequest;
		httpResult.httpResponse = response;
		return httpResult;
	}
	
	/**
	 * 执行http请求,得到响应
	 * 
	 * @param request
	 * @param proxy
	 * @return HttpResponse
	 * @throws ClientProtocolException
	 * @throws IOException
	 */
	public static HttpResponse executeRequest(HttpUriRequest request, Proxy proxy) throws ClientProtocolException, IOException {
		if (request == null) {
			throw new IllegalArgumentException("Request must not be null.");
		}

		if (null == proxy) {
			return httpClient.execute(request);
		}

		AbstractHttpClient proxyHttpClient = HttpClientFactory.getNewHttpClient();
		// 设置代理主机和端口
		HttpHost proxyHost = new HttpHost(proxy.getIp(), proxy.getPort());
		proxyHttpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY,	proxyHost);

		// 设置用户名、密码
		String userName = proxy.getUserName();
		String passWord = proxy.getPassWord();
		if (userName != null && !"".equals(userName)) {
			if (null == passWord) {
				passWord = "";
			}
			UsernamePasswordCredentials cred = new UsernamePasswordCredentials(userName, passWord);
			proxyHttpClient.getCredentialsProvider().setCredentials(AuthScope.ANY, cred);
		}

		HttpResponse httpResponse =  proxyHttpClient.execute(request);
		return httpResponse;
	}
	
	/**
	 * 得到响应头content-type的值
	 * 
	 * @param response
	 * @return 成功:content-type的值, 失败:null
	 */
	public static String getContentType(HttpResponse response) {
		Header contentTypeHeader = response.getFirstHeader("Content-Type");
		if (contentTypeHeader != null) {
			return contentTypeHeader.getValue();
		}
		return null;
	}
	
	/**
	 * 将流的内容读到byte[]当中
	 * 
	 * @param instream
	 * @param contentLength，流的内容长度，未知长度,可传负值
	 * @return 成功:byte[],失败:null
	 * @throws IOException 如果在读流时发生了错误
	 */
	private static byte[] readToByteArray(final InputStream instream, long contentLength) throws IOException {
		if (instream == null) {
			return null;
		}
		if (contentLength > Integer.MAX_VALUE) {
			return null;
		}
		InputStream in = new BufferedInputStream(instream);

		// 未知长度
		if (contentLength < 0) {
			contentLength = 8 * 1024;
		}
		ByteArrayBuffer buffer = new ByteArrayBuffer((int) contentLength);
		byte[] tmp = new byte[4096];
		int len;
		while ((len = in.read(tmp)) != -1) {
			buffer.append(tmp, 0, len);
		}
		return buffer.toByteArray();
	}

	/**
	 * 关闭响应流
	 * 
	 * @param httpResponse
	 * @param url
	 */
	private static void closeResponse(HttpResponse httpResponse, String url) {
		if (null == httpResponse || null == httpResponse.getEntity()) {
			return;
		}

		try {
			httpResponse.getEntity().getContent().close();// 关闭流,释放连接;
		} catch (Exception e) {
			log.error("url:" + url);
			log.error(e.getMessage(), e);
		}
	}
	
	private static class HttpResult {
		private String finalUrl; // 最终url
		private HttpUriRequest httpRequest;
		private HttpResponse httpResponse;
	}
	
	/**
	 * 构造http请求
	 * 
	 * @param httpSnatchParam
	 * @return 失败:null
	 */
	private static HttpUriRequest constructHttpRequest(HttpSnatchParam httpSnatchParam) {
		// http参数
		String url = httpSnatchParam.getUrl();
		List<Header> addRequestHeaders = httpSnatchParam.getRequestHeaders();
		List<NameValuePair> postParams = httpSnatchParam.getPostParams();
		String postParamsEncoding = httpSnatchParam.getPostParamsEncoding();

		if (null == url || !url.toLowerCase().startsWith("http")) {
			log.error("illegal http url:[{}]", url);
			return null;
		}

		// 预处理url
		url = StringUtils.hexToStr(url);
		if (httpSnatchParam.isNeedEncodeUrl()) {
			String urlEncoding = httpSnatchParam.getUrlEncoding();
			if (urlEncoding == null || urlEncoding.length() == 0) {
				urlEncoding = "utf-8"; // 默认为"utf-8"
			}
			log.debug("url=" + url);
			url = URLUtil.encode(url, urlEncoding);
			log.debug("encoded url=" + url);
		}

		// 构造http请求
		HttpUriRequest httpRequest = null;
		if (postParams != null && postParams.size() > 0) { // post方法
			if (null == postParamsEncoding || "".equals(postParamsEncoding)) {
				log.error("Don't provide post encoding parameter!");
				return null;
			}
			HttpPost httPost = new HttpPost(url);
			try {
				httPost.setEntity(new UrlEncodedFormEntity(postParams,
						postParamsEncoding));
			} catch (Exception e) {
				log.error("postParamsEncoding:" + postParamsEncoding + " for url:" + url);
				log.error(e.getMessage(), e);
				return null;
			}
			httpRequest = httPost;
		} else { // get方法
			HttpGet httpGet = new HttpGet(url);
			httpRequest = httpGet;
		}

		// 添加额外请求头
		if (null != addRequestHeaders && addRequestHeaders.size() > 0) {
			for (Header header : addRequestHeaders) {
				httpRequest.addHeader(header);
			}
		}

		return httpRequest;
	}
}
